import requests
from bs4 import BeautifulSoup
from fake_useragent import UserAgent
import time
import random
import logging

class BaseScraper:
    def __init__(self):
        self.ua = UserAgent()
        self.session = requests.Session()
        self.logger = logging.getLogger(self.__class__.__name__)
        
    def get_headers(self):
        return {
            'User-Agent': self.ua.random,
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'Accept-Encoding': 'gzip, deflate',
            'Connection': 'keep-alive',
        }
    
    def get(self, url, params=None):
        """发送GET请求并返回BeautifulSoup对象"""
        try:
            headers = self.get_headers()
            response = self.session.get(url, headers=headers, params=params, timeout=10)
            response.raise_for_status()
            response.encoding = 'utf-8'
            
            # 随机延迟，避免被封
            time.sleep(random.uniform(1, 3))
            
            return BeautifulSoup(response.text, 'lxml')
        except Exception as e:
            self.logger.error(f"请求失败: {url}, 错误: {str(e)}")
            return None
    
    def search(self, keyword):
        """子类必须实现的方法"""
        raise NotImplementedError
